{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "import torch\n",
    "import json\n",
    "\n",
    "eval_instructions = json.load(open(\"dataset/Arxiv2023_instruction_stage2/instructions.json\"))\n",
    "eval_data = torch.load(\"dataset/Arxiv2023_instruction_stage2/test_data_list_8192_llama3.pt\")\n",
    "\n",
    "eval_instructions = {\n",
    "    str(entry[\"id\"]): entry for entry in eval_instructions\n",
    "}\n",
    "\n",
    "output_prompts = []\n",
    "for entry in eval_data:\n",
    "    id = str(entry[\"id\"])\n",
    "    truth_i = entry[\"ground_truth\"]\n",
    "\n",
    "    prompt_i = eval_instructions[id]\n",
    "\n",
    "    input_i = prompt_i[\"conversations\"][0][\"value\"]\n",
    "    input_i = input_i.replace(\"For each paper, a heterogeneous knowledge graph is constructed to reflect the concepts and key notes in the title and abstract. In the knowledge graph, there are five types of nodes: namely: paper, research_background, research_question, methodology, key_results, keyword. The relationships (meta paths) between different nodes include: [paper has_property research_background], [paper has_property research_question], [paper has_property methodology], [paper has_property key_results], [paper has_keyword keyword], [research_background has_keyword keyword], [research_question has_keyword keyword], [methodology has_keyword keyword], [key_results has_keyword keyword].\\n\", \"\")\n",
    "    input_i_parts = input_i.split(\"Heterogeneous Graph: \\\"paper\\\" node: <graph>;\")\n",
    "    input_i = input_i_parts[0] + \"Based on the title, abstract and the provided knowledge graph, which Arxiv category should the paper be categorized into?\" + input_i_parts[1].split(\"Based on the title, abstract and the provided knowledge graph, which Arxiv category should the paper be categorized into?\")[1]\n",
    "    input_i = input_i.replace(\" and the provided knowledge graph\", \"\")\n",
    "\n",
    "    # print(input_i)\n",
    "    \n",
    "    output_i = {\n",
    "        \"id\": id,\n",
    "        \"prompt\": [\n",
    "            {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": input_i\n",
    "            }\n",
    "        ],\n",
    "        \"ground_truth\": truth_i\n",
    "    }\n",
    "    output_prompts.append(output_i)\n",
    "\n",
    "json.dump(output_prompts, open(\"dataset/Arxiv2023_instruction_stage2/eval_prompts.json\", \"w\"), indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning\n"
     ]
    }
   ],
   "source": [
    "%cd /hpc2hdd/home/xzou428/Yuhao/HiGPT-tune-lightning/\n",
    "import torch\n",
    "import json\n",
    "import re\n",
    "\n",
    "eval_instructions = json.load(open(\"dataset/ICLR_peer_review_stage2/instructions.json\"))\n",
    "eval_data = torch.load(\"dataset/ICLR_peer_review_stage2/test_data_list_8192_llama3.pt\")\n",
    "\n",
    "eval_instructions = {\n",
    "    str(entry[\"id\"]): entry for entry in eval_instructions\n",
    "}\n",
    "\n",
    "# use regex to match patterns like \\\"xxxx <graph>\\\"\n",
    "remove_pattern = re.compile(r\"\\\".*?<graph>\\\"\")\n",
    "\n",
    "output_prompts = []\n",
    "for entry in eval_data:\n",
    "    id = str(entry[\"id\"])\n",
    "    truth_i = entry[\"ground_truth\"]\n",
    "\n",
    "    prompt_i = eval_instructions[id]\n",
    "\n",
    "    input_i = prompt_i[\"conversations\"][0][\"value\"]\n",
    "    input_i = input_i.replace(\"For each paper together with its reviews, a heterogeneous knowledge graph is constructed to reflect the concepts and key notes in the whole content. In the knowledge graph, there are several types of nodes: namely: paper, strengths_in_review_n, weaknesses_in_review_n, keyword, with n subject to the number of reviews. The relationships (meta paths) between different nodes include: [paper has_property strengths_in_review_n], [paper has_property weaknesses_in_review_n], [strengths_in_review_n has_keyword keyword], [weaknesses_in_review_n has_keyword keyword], [paper has_keyword keyword].\\n\", \"\")\n",
    "    input_i = input_i.replace(\" and the knowledge graph\", \"\")\n",
    "    input_i = input_i.replace(\"\\\"paper\\\" node in the knowledge graph: <graph>\", \"\")\n",
    "    input_i = re.sub(remove_pattern, \"\", input_i)\n",
    "    # print(input_i)\n",
    "    # break\n",
    "    \n",
    "    output_i = {\n",
    "        \"id\": id,\n",
    "        \"prompt\": [\n",
    "            {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": input_i\n",
    "            }\n",
    "        ],\n",
    "        \"ground_truth\": truth_i\n",
    "    }\n",
    "    output_prompts.append(output_i)\n",
    "\n",
    "json.dump(output_prompts, open(\"dataset/ICLR_peer_review_stage2/eval_prompts.json\", \"w\"), indent=4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.14 ('yuh')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "92619f85c62eb73280c07ca2268c8e47b90999a589aa097a1a08a504bd2fb2c6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
